package main

import (
	//"bufio"
	"bytes"
	"code.google.com/p/go.text/encoding/simplifiedchinese"
	"code.google.com/p/go.text/transform"
	"flag"
	"fmt"
	goq "github.com/PuerkitoBio/goquery"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

var (
	main_domain = "" //主域名
	LOG         = "anjuke"
	ErrLOG      = "anjuke"
)

func main() {
	citycode := flag.String("citycode", "wuhan", "请输入城市编号，例如 --citycode='shanghai'")
	flag.Parse()

	main_domain = "http://" + *citycode + ".anjuke.com"
	LOG = *citycode + "." + LOG
	ErrLOG = *citycode + "." + ErrLOG

	getDistrict("/community")
	fmt.Println("done.")
}

//获取区域
func getDistrict(url string) {
	tmpurl := fmt.Sprintf("%s%s", main_domain, url)
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog("抓取区域列表失败 [" + tmpurl + "] [" + e.Error() + "]")
		return
	}

	doc.Find(".con").Eq(0).Find("a").Each(func(i int, sec *goq.Selection) {
		var (
			district, link string
		)
		link, ok := sec.Attr("href")
		if ok {
			district = filterStr(sec.Text())
			link = filterStr(link)
			getBlock(link, district)
		}
	})
}

//获取板块列表
func getBlock(url string, district string) {
	tmpurl := url
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog("抓取热点列表失败 [" + tmpurl + "] [" + e.Error() + "]")
		return
	}

	doc.Find(".sub a").Each(func(i int, sec *goq.Selection) {
		var (
			block, link string
		)
		link, ok := sec.Attr("href")
		if ok {
			block = filterStr(sec.Text())
			link = filterStr(link)
			getCommunitylist(link, block, district)
		}
	})
}

//获取楼盘列表
func getCommunitylist(url string, block, district string) {
	tmpurl := url
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog("抓取楼盘列表失败 [" + tmpurl + "] [" + e.Error() + "]")
		return
	}

	//获取分页
	tmp_recordCount := filterStr(doc.Find(".PL_1 em").Text())
	recordCount, e := strconv.Atoi(tmp_recordCount)
	if e != nil {
		errlog("抓取楼盘列表页数失败 [" + tmpurl + "] [" + e.Error() + "]")
		return
	}
	pageSize := 10
	pageCount := recordCount / pageSize
	if pageCount*pageSize < recordCount {
		pageCount = pageCount + 1
	}

	communitylist_chan := make(chan int, runtime.NumCPU())

	for i := 1; i <= pageCount; i++ {
		community_url := strings.Replace(tmpurl, "QQpZ1", "", -1)
		community_url = fmt.Sprintf("%sQQpZ%d", community_url, i)
		go getCommunityInfo(communitylist_chan, community_url, block, district)
		<-communitylist_chan
	}
	time.Sleep(time.Second * 10)
}

//获取楼盘信息
func getCommunityInfo(communitylist_chan chan int, tmpurl string, block, district string) {
	defer func() { communitylist_chan <- 1 }()
	doc, e := getDoc(tmpurl)
	if e != nil {
		errlog("抓取楼盘列表分页失败 [" + tmpurl + "] [" + e.Error() + "]")
		return
	}
	doc.Find(".list .list_item").Each(func(i int, sec *goq.Selection) {
		var (
			communityname string //楼盘名
			communityaddr string //楼盘地址
			avgprice      string //均价
			pricebrand    string //价格走势
		)

		communityname = filterStr(sec.Find(".t_b").Text())
		if communityname == "" {
			communityname = "--"
		}
		communityaddr = filterStr(sec.Find(".t_b").Next().Text())
		communityaddr = strings.Replace(communityaddr, "["+district+" "+block+"]", "", -1)
		if communityaddr == "" {
			communityaddr = "--"
		}
		avgprice = filterStr(sec.Find(".price .sp1").Text())
		if avgprice == "" {
			avgprice = "--"
		}
		pricebrand = filterStr(sec.Find(".price i").Text())
		if pricebrand == "" {
			pricebrand = "--"
		}

		str := district
		str += "||" + block
		str += "||" + communityname
		str += "||" + communityaddr
		str += "||" + avgprice
		str += "||" + pricebrand
		outlog(str)
	})
}

//获取doc
func getDoc(url string) (*goq.Document, error) {
	res, e := getResp(url)
	if e != nil {
		return nil, e
	}
	doc, e := goq.NewDocumentFromResponse(res)
	if e != nil {
		return nil, e
	}
	return doc, nil
}

//获取response
func getResp(url string) (*http.Response, error) {
	req, e := http.NewRequest("GET", url, nil)
	req.Header.Add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	//req.Header.Add("Accept-Encoding", "gzip,deflate,sdch")
	req.Header.Add("Accept-Language", "zh-CN,zh;q=0.8")
	req.Header.Add("Cache-Control", "max-age=0")
	req.Header.Add("Proxy-Connection", "keep-alive")
	req.Header.Add("User-Agent", "Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/31.0.1650.63 Safari/537.36")
	c := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				deadline := time.Now().Add(25 * time.Second)
				c, err := net.DialTimeout(netw, addr, time.Second*20)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	var resp *http.Response
	//尝试三次
	for retry := 0; retry < 3; retry++ {
		resp, e = c.Do(req)
		if e != nil {
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if e != nil {
		return nil, e
	}
	return resp, nil
}

//gbk->utf8
func G2U(s string) string {
	ret, e := gbktoutf8([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

//utf8->gbk
func U2G(s string) string {
	ret, e := utf8togbk([]byte(s))
	if e != nil {
		return ""
	}
	return string(ret)
}

func gbktoutf8(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewDecoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func utf8togbk(s []byte) ([]byte, error) {
	reader := transform.NewReader(bytes.NewReader(s), simplifiedchinese.GBK.NewEncoder())
	d, e := ioutil.ReadAll(reader)
	if e != nil {
		return nil, e
	}
	return d, nil
}

func filterStr(str string) string {
	str = strings.Replace(str, "\r\n", "", -1)
	str = strings.Replace(str, "\r", "", -1)
	str = strings.Replace(str, "\n", "", -1)
	str = strings.Replace(str, "\t", "", -1)
	str = strings.Replace(str, "|", "", -1)
	str = strings.Replace(str, ",", "", -1)
	str = strings.TrimSpace(str)
	return str
}

func s(a ...interface{}) {
	fmt.Println(a)
}

//outlog
func outlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_log.txt", pwd, LOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}

//errlog
func errlog(str string) {
	pwd, _ := os.Getwd()
	outlog := fmt.Sprintf("%s/%s_err.txt", pwd, ErrLOG)
	file, err := os.OpenFile(outlog, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0600)
	defer file.Close()
	if err != nil {
		return
	}
	str += "\n"
	file.WriteString(str)
}
